# -*- coding: utf-8 -*-
# @Time    : 2024/05/07 17:08
# @Author  : Mr.su
# @FileName: shop-msgm.py
# @FileDesc: 
from CollectSpiders.settings import LOG_FILE_PATH
from CollectSpiders.toots.methods import make_md5
from CollectSpiders.toots.connects import RedisClient
import scrapy, json, datetime, urllib.parse, logging, re


# noinspection PyAbstractClass,PyMethodMayBeStatic
class CrawlSpider(scrapy.Spider):
    name, domain = 'shop-msgm', 'www.shop-msgm.com'
    redisClient = RedisClient()
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
    }
    custom_settings = {
        'CONCURRENT_REQUESTS': 2,
        'LOG_FILE': LOG_FILE_PATH, 'LOG_LEVEL': 'WARNING'
    }

    def __init__(self, data=None, *args, **kwargs):
        """
        传入自定义参数
        :param data: 数据列表
        :param args:
        :param kwargs:
        """
        super(CrawlSpider, self).__init__(*args, **kwargs)
        self.data = json.loads(urllib.parse.unquote(data)) if data else {
            "pid": "1700", "status": 1, "wait_time": 3600, "brand": "shop-msgm",
            "childs": {
                "1701": {
                    "sid": "1701", "label": "女新品", "upload": 1, "status": 1, "webtype": "taobao",
                    "url": "https://www.shop-msgm.com/en-cn/collections/women-new-in"
                }
            }
        }

    def start_requests(self):
        column = list(self.data['childs'].values())[0]
        if column['status'] == 1:
            yield scrapy.Request(column['url'], headers=self.headers, callback=self.process_home)

    def process_home(self, response):
        data_count = ''.join(response.xpath('//*[@id="prodCounter"]/b/text()').extract_first())
        pg_count = int(data_count) // 20 if int(data_count) % 20 == 0 else int(data_count) // 20 + 1
        for pg in range(1, pg_count + 1):
            column = list(self.data['childs'].values())[0]
            new_url = column['url'] + '?page={}'
            yield scrapy.Request(
                new_url.format(pg), headers=self.headers, callback=self.process_lis,
            )

    def process_lis(self, response):
        detail_lis = re.findall(r'"product_url": "(.*?)",', response.text)
        for data in detail_lis:
            detail_url = 'https://www.shop-msgm.com{}.json'.format(data)
            result = self.redisClient.conn.sadd('products', make_md5(detail_url))
            if not result:
                break
            yield scrapy.Request(detail_url, headers=self.headers, callback=self.process_detail)

    def process_detail(self, response):
        column = list(self.data['childs'].values())[0]
        js = json.loads(response.text)
        img_lis = [i['src'] for i in js['product']['images']]
        product = {
            '_id': make_md5(img_lis[0]),
            'pid': self.data['pid'],
            'sid': column['sid'],
            'status': '0',  # 0:未下载  1:已下载  6:下载失败
            'webtype': column['webtype'],
            'url': response.url,
            'time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            'brand': self.data['brand'],
            'image': img_lis,
            'domain': self.domain,
            'label': column['label'],
            'upload': column['upload'],
            'name': js['product']['title']
        }
        logging.warning('<{}>: 数据id: {}'.format(self.domain, product['_id']))
        yield product
